﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;
using System.Xml;
using System.Security.Policy;

namespace CreateCorpus
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
            GetFinalSemcorVi();
        }

        //lấy nội dung của một trang web
        protected string GetWebContent(string url)
        {
            string pageContent = null;
            HttpWebRequest myReq = (HttpWebRequest)WebRequest.Create(url);
            HttpWebResponse myres = (HttpWebResponse)myReq.GetResponse();

            using (StreamReader sr = new StreamReader(myres.GetResponseStream()))
            {
                pageContent = sr.ReadToEnd();
            }
            return pageContent;
        }


        //lọc ra các đường dẫn chứa cặp file vn và en.
        private void Filter_Type_2()
        {
            string path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\CreateCorpus\links_vi_en.txt";
            if (File.Exists(path))
                File.Delete(path);

            string path1 = @"http://www.ibm.com/developerworks/vn/views/opensource/libraryview.jsp?site_id=70&contentarea_by=Tất cả các vùng&sort_by=&sort_order=2&start={0}&end={1}&topic_by=Tất cả chủ đề về &product_by=&type_by=Bài báo&show_abstract=true&search_by=&industry_by=&series_title_by=";
            List<string> links = new List<string>();
            int to = 276;
            int size = 100;
            string web_content, link_vi, link_en;
            List<string> Error = new List<string>();

            Regex re_table = new Regex("(<table (.|\n)+?ibm-data-table (.|\n)+?</table>)");
            Regex re_link_vi = new Regex("<td><a href=\"(.+)\">");

            for (int i = 1; i < to; i += size)
            {
                string url = String.Format(path1, i, i + size);
                web_content = GetWebContent(url);
                string table = re_table.Match(web_content).Value;

                Match ma = re_link_vi.Match(table);
                while (ma.Success)
                {
                    link_vi = String.Format("{0}", ma.Groups[1].Value);
                    if (!links.Contains(link_vi))
                        links.Add(link_vi);
                    // else
                    //     MessageBox.Show(link);
                    ma = ma.NextMatch();
                }
            }

            Regex re_link_en1 = new Regex("<br class=\"ibm-ind-link\".+?>.+?<a.+?href=\"(.+?.html).*?\">tại đây</a>");
            Regex re_link_en2 = new Regex("<br class=\"ibm-ind-link\".+?>.+?<a.+?href=\"(.+?.html).*?\">tiếng Anh</a>");
            for (int i = 0; i < links.Count; i++)
            {
                try
                {
                    web_content = GetWebContent(links[i]);

                    link_en = re_link_en1.Match(web_content).Groups[1].Value;
                    if (link_en == "")
                        link_en = re_link_en2.Match(web_content).Groups[1].Value;

                    if (link_en == "")
                    {
                        Error.Add("<no>" + links[i] + "</no>");
                    }
                    else
                    {
                        links[i] = String.Format("<link>    <vi>{0}</vi>    <en>{1}</en>    </link>", links[i], link_en);
                        File.AppendAllText(path, links[i] + Environment.NewLine);
                    }
                }
                catch
                {
                    Error.Add(links[i]);
                }
            }

            string path_error = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\CreateCorpus\links_vi_en_error.txt";
            File.WriteAllText(path_error, String.Join(Environment.NewLine, Error.ToArray()));
            MessageBox.Show("Done");
        }


        //lọc lấy các paragraph kiểu cũ
        //private void button1_Click(object sender, EventArgs e)
        //{
        //    string path = @"C:\Documents and Settings\bRea.K\Desktop\source-en.txt";
        //    string content = File.ReadAllText(path);

        //    Regex re_maincontent = new Regex("<!-- MAIN_COLUMN_CONTENT_BEGIN -->((.|\n)*?)<!-- MAIN_COLUMN_CONTENT_END -->");
        //    Regex re_a = new Regex("<a(.|\n)+?>((.|\n)+?)</a>");
        //    Regex re_span = new Regex("<span(.|\n)+?>((.|\n)+?)</span>");
        //    Regex re_p = new Regex("<p>([^<]+)</p>");

        //    string maincontent = re_maincontent.Match(content).Value;
        //    string output = re_a.Replace(maincontent, "$2");
        //    output = re_span.Replace(output, "$2");

        //    Match[] arr = new Match[100];
        //    re_p.Matches(output).CopyTo(arr,0);

        //    path +=".p.txt";
        //    foreach (Match ma in arr)
        //    {
        //        if (ma == null)
        //            break;
        //        File.AppendAllText(path, "<p>"+ma.Groups[1].Value+"</p>" + Environment.NewLine);
        //    }

        //}


        //lọc các paragraph kiểu mới
        private void TachDoan()
        {
            string path = @"C:\Documents and Settings\bRea.K\Desktop\sourcevn-full1.txt";
            string content = File.ReadAllText(path);

            Regex re_maincontent = new Regex("<!-- MAIN_COLUMN_CONTENT_BEGIN -->((.|\n)*?)<!-- MAIN_COLUMN_CONTENT_END -->");
            string maincontent = re_maincontent.Match(content).Value;

            WebBrowser webBrowser1 = new WebBrowser();
            webBrowser1.Navigate("about:blank");
            webBrowser1.Document.Write(maincontent);
            var items = webBrowser1.Document.GetElementsByTagName("p");

            string source = "";
            foreach (HtmlElement item in items)
            {
                source += String.Format("<p>{0}</p>\n", item.InnerText);
            }

            path += "done.html";
            File.WriteAllText(path, source);
        }

        //hàm tách thử ra các câu
        private void TachCau()
        {
            string path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\vn.html";
            string content = File.ReadAllText(path);

            WebBrowser webBrowser1 = new WebBrowser();
            webBrowser1.Navigate("about:blank");
            webBrowser1.Document.Write(content);

            var items = webBrowser1.Document.GetElementsByTagName("p");
            string source = "";
            foreach (HtmlElement item in items)
            {
                var re = Regex.Split(item.InnerText, "[.?:!]");

                foreach (var sen in re)
                {
                    File.AppendAllText(path + ".sen.txt", "<s>" + sen + "</s>" + Environment.NewLine);
                }
            }
        }

        //hàm chính đầu tiên, lọc ra tập hợp các file en và vi, đưa vào thư mục source tương ứng. Các link hư được đưa vào file error.txt
        private void SaveContent()
        {
            string path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\CreateCorpus\links_vi_en.txt";
            string content = File.ReadAllText(path);
            //WebBrowser webBrowser1 = new WebBrowser();
            //webBrowser1.Navigate("about:blank");
            //webBrowser1.Document.Write(content);

            string source_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\source\";
            string error_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\error.txt";
            string done_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\done.txt";

            string link_vi = "";
            string link_en = "";
            string content_vi = "";
            string content_en = "";
            int i = 0;
            Regex re_link = new Regex("<link>(.+?)</link>");
            Regex re_vi = new Regex("<vi>(.+?)</vi>");
            Regex re_en = new Regex("<en>(.+?)</en>");

            MatchCollection matches = re_link.Matches(content);
            foreach (Match item in matches)
            {
                try
                {
                    link_vi = re_vi.Match(item.Groups[1].Value).Groups[1].Value;
                    link_en = re_en.Match(item.Groups[1].Value).Groups[1].Value;

                    content_vi = GetWebContent(link_vi);
                    content_en = GetWebContent(link_en);

                    File.WriteAllText(source_path + i + "_vi.html", content_vi);
                    File.WriteAllText(source_path + i + "_en.html", content_en);
                    i++;
                    File.AppendAllText(done_path, String.Format("<link_done><vi>{0}</vi><en>{1}</en></link_done>", link_vi, link_en) + Environment.NewLine);

                }
                catch (Exception ex)
                {
                    File.AppendAllText(error_path, String.Format("<link_fail><vi>{0}</vi><en>{1}</en><error>{2}</error></link_fail>", link_vi, link_en, ex.Message) + Environment.NewLine);
                }
            }
        }

        //hàm tách đoạn đầu tiên sử dụng cho tập file lấy về, được lưu trữ trong source
        private void GetParagraph()
        {
            string source_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\source\{0}_en.html";
            string source_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\source\{0}_vi.html";

            string paragraph_done_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_done\{0}_p_en.txt";
            string paragraph_done_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_done\{0}_p_vi.txt";

            string paragraph_error_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_error\{0}_p_en.txt";
            string paragraph_error_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_error\{0}_p_vi.txt";

            string paragraph_error_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\error_p.txt";

            string content_vi = "";
            string content_en = "";

            Regex re_maincontent = new Regex("<!-- MAIN_COLUMN_CONTENT_BEGIN -->((.|\n)*?)<!-- MAIN_COLUMN_CONTENT_END -->");

            int done = 0;
            int error = 0;


            for (int i = 0; i <= 256; i++)
            {
                try
                {
                    content_en = File.ReadAllText(string.Format(source_en_path, i));
                    content_vi = File.ReadAllText(string.Format(source_vi_path, i));

                    //for vi
                    content_vi = re_maincontent.Match(content_vi).Value;
                    WebBrowser webBrowser1 = new WebBrowser();
                    webBrowser1.Navigate("about:blank");
                    webBrowser1.Document.Write(content_vi);
                    var items = webBrowser1.Document.GetElementsByTagName("p");

                    content_vi = "";
                    foreach (HtmlElement item in items)
                        content_vi += String.Format("<p>{0}</p>\n", item.InnerText);

                    //for en
                    content_en = re_maincontent.Match(content_en).Value;
                    WebBrowser webBrowser2 = new WebBrowser();
                    webBrowser2.Navigate("about:blank");
                    webBrowser2.Document.Write(content_en);
                    var items1 = webBrowser2.Document.GetElementsByTagName("p");

                    content_en = "";
                    foreach (HtmlElement item in items1)
                        content_en += String.Format("<p>{0}</p>\n", item.InnerText);

                    if (items1.Count == items.Count)
                    {
                        File.WriteAllText(string.Format(paragraph_done_vi_path, done), content_vi);
                        File.WriteAllText(string.Format(paragraph_done_en_path, done), content_en);
                        done++;
                    }
                    else
                    {
                        File.WriteAllText(string.Format(paragraph_error_vi_path, error), content_vi, Encoding.Unicode);
                        File.WriteAllText(string.Format(paragraph_error_en_path, error), content_en, Encoding.Unicode);
                        error++;
                    }
                }
                catch (Exception ex)
                {
                    File.AppendAllText(paragraph_error_path, string.Format("<file><number>{0}</number><error>{1}</error></file>", i, ex.Message) + Environment.NewLine);
                }
            }
        }

        //hàm viết tạm để chuyển các file trong error vào done (các file trong error đã được chỉnh sửa cho khớp lại)
        private void Move()
        {
            string paragraph_done_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_done\{0}_p_en.txt";
            string paragraph_done_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_done\{0}_p_vi.txt";

            string paragraph_error_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_error\{0}_p_en.txt";
            string paragraph_error_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_error\{0}_p_vi.txt";

            int j = 195;
            string content_en = "";
            string content_vi = "";

            for (int i = 1; i <= 61; i++)
            {

                content_en = File.ReadAllText(string.Format(paragraph_error_en_path, i));
                content_vi = File.ReadAllText(string.Format(paragraph_error_vi_path, i));

                WebBrowser webBrowser1 = new WebBrowser();
                webBrowser1.Navigate("about:blank");
                webBrowser1.Document.Write(content_vi);
                var items = webBrowser1.Document.GetElementsByTagName("p");

                WebBrowser webBrowser2 = new WebBrowser();
                webBrowser2.Navigate("about:blank");
                webBrowser2.Document.Write(content_en);
                var items1 = webBrowser2.Document.GetElementsByTagName("p");

                if (items.Count == items1.Count)
                {
                    File.WriteAllText(string.Format(paragraph_done_vi_path, j), content_vi);
                    File.WriteAllText(string.Format(paragraph_done_en_path, j), content_en);
                    j++;
                }
                else
                {

                }
            }
        }

        //đếm có bao nhiêu câu
        private void GetSentence()
        {
            string paragraph_done_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_done\{0}_p_en.txt";
            string paragraph_done_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_done\{0}_p_vi.txt";

            string paragraph_error_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_error\{0}_p_en.txt";
            string paragraph_error_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_error\{0}_p_vi.txt";

            string content_en = "";
            string content_vi = "";

            for (int i = 0; i < 256; i++)
            {
                content_en = File.ReadAllText(string.Format(paragraph_done_en_path, i));
                content_vi = File.ReadAllText(string.Format(paragraph_done_vi_path, i));

                WebBrowser webBrowser1 = new WebBrowser();
                webBrowser1.Navigate("about:blank");
                webBrowser1.Document.Write(content_vi);
                var items = webBrowser1.Document.GetElementsByTagName("p");

                WebBrowser webBrowser2 = new WebBrowser();
                webBrowser2.Navigate("about:blank");
                webBrowser2.Document.Write(content_en);
                var items1 = webBrowser2.Document.GetElementsByTagName("p");



                for (int j = 0; j < items1.Count; j++)
                {
                    try
                    {
                        string[] sentense_en = Regex.Split(items[j].InnerText, "[.?:!]");
                        string[] sentense_vi = Regex.Split(items1[j].InnerText, "[.?:!]");

                        if (sentense_en.Length != sentense_vi.Length)
                        {
                            // count++;
                        }
                        else
                        {
                            //  count1 += sentense_vi.Length;
                        }
                    }
                    catch (Exception ex)
                    {

                    }
                }
            }
        }

        //gán số thứ tự cho các para
        private void GetParagraphWithNumber()
        {
            string paragraph_done_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_done\{0}_p_en.txt";
            string paragraph_done_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_done\{0}_p_vi.txt";

            string paragraph_error_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_checked\{0}_p_en.txt";
            string paragraph_error_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_checked\{0}_p_vi.txt";

            string content_en = "";
            string content_vi = "";

            int j = 0;

            for (int i = 0; i < 256; i++)
            {
                content_en = File.ReadAllText(string.Format(paragraph_done_en_path, i));
                content_vi = File.ReadAllText(string.Format(paragraph_done_vi_path, i));

                WebBrowser webBrowser1 = new WebBrowser();
                webBrowser1.Navigate("about:blank");
                webBrowser1.Document.Write(content_vi);
                var items = webBrowser1.Document.GetElementsByTagName("p");

                content_vi = "";
                for (int m = 0; m < items.Count; m++)
                {
                    content_vi += String.Format("<p={0}>{1}</p>\n", m, items[m].InnerText);
                }

                WebBrowser webBrowser2 = new WebBrowser();
                webBrowser2.Navigate("about:blank");
                webBrowser2.Document.Write(content_en);
                var items1 = webBrowser2.Document.GetElementsByTagName("p");

                content_en = "";
                for (int m = 0; m < items1.Count; m++)
                {
                    content_en += String.Format("<p={0}>{1}</p>\n", m, items1[m].InnerText);
                }
                if (items.Count == items1.Count)
                {
                    File.WriteAllText(string.Format(paragraph_error_vi_path, j), content_vi);
                    File.WriteAllText(string.Format(paragraph_error_en_path, j), content_en);
                    j++;
                }
            }
        }

        //sửa lại gán số thứ tự với ID
        private void GetSentenceWithNumberID()
        {
            string paragraph_done_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_mini\{0}_p_en.txt";
            string paragraph_done_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_mini\{0}_p_vi.txt";

            string paragraph_error_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_id_after_except\{0}_p_en.txt";
            string paragraph_error_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_id_after_except\{0}_p_vi.txt";

            string content_en = "";
            string content_vi = "";

            int j = 0;

            for (int i = 0; i < 256; i++)
            {
                try
                {
                    content_en = File.ReadAllText(string.Format(paragraph_done_en_path, i));
                    content_vi = File.ReadAllText(string.Format(paragraph_done_vi_path, i));

                    content_en = Regex.Replace(content_en, "<p=([0-9]{1,3})>(.*?)</p>", "<p id=$1>$2</p>");
                    content_vi = Regex.Replace(content_vi, "<p=([0-9]{1,3})>(.*?)</p>", "<p id=$1>$2</p>");

                    File.WriteAllText(string.Format(paragraph_error_vi_path, j), content_vi);
                    File.WriteAllText(string.Format(paragraph_error_en_path, j), content_en);
                    j++;
                }
                catch
                { }
            }
        }

        //gán id + thẻ sentense
        private void GetParagraphWithNumberID()
        {
            string paragraph_done_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\ngulieu\ENcorpus.final.txt";
            string paragraph_done_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\ngulieu\VIcorpus.final.txt";

            string paragraph_error_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\ngulieu\ENcorpus.final.tag";
            string paragraph_error_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\ngulieu\VIcorpus.final.tag";

            string content_en = "";
            string content_vi = "";


            content_en = File.ReadAllText(paragraph_done_en_path);
            content_vi = File.ReadAllText(paragraph_done_vi_path);
            string output_en = "", output_vi = "";
            string output_en_min = "", output_vi_min = "";

            string[] arr_en = Regex.Split(content_en, "\r\n");
            string[] arr_vi = Regex.Split(content_vi, "\r\n");

            for (int i = 0; i < arr_en.Length; i++)
            {

                if ((arr_en[i] == null && arr_vi[i] == null) || (arr_vi[i].Length <= 30 && arr_en[i].Length <= 30))
                {
                    output_en_min += string.Format("<div class='sentence' id={0}>{1}</div>", i, arr_en[i]);
                    output_vi_min += string.Format("<div class='sentence' id={0}>{1}</div>", i, arr_vi[i]);
                }
                else
                {
                    output_en += string.Format("<div class='sentence' id={0}>{1}</div>", i, arr_en[i]);
                    output_vi += string.Format("<div class='sentence' id={0}>{1}</div>", i, arr_vi[i]);
                }
            }

            File.WriteAllText(paragraph_error_vi_path, output_en);
            File.WriteAllText(paragraph_error_en_path, output_vi);
            //File.WriteAllText(paragraph_error_vi_path+".min", output_en_min);
            //File.WriteAllText(paragraph_error_en_path+".min", output_vi_min);

        }

        //sửa lại bỏ hết các thẻ <p>
        private void GetParagraphWithoutTag()
        {
            string paragraph_done_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_new\{0}_p.en";
            string paragraph_done_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_new\{0}_p.vi";

            string paragraph_error_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_new_without_tag\{0}_p.en";
            string paragraph_error_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_new_without_tag\{0}_p.vi";

            string content_en = "";
            string content_vi = "";

            int j = 0;

            for (int i = 0; i < 256; i++)
            {
                try
                {
                    content_en = File.ReadAllText(string.Format(paragraph_done_en_path, i));
                    content_vi = File.ReadAllText(string.Format(paragraph_done_vi_path, i));

                    content_en = Regex.Replace(content_en, "<p id=([0-9]{1,3})>(.*?)</p>", "$2" + Environment.NewLine);
                    content_vi = Regex.Replace(content_vi, "<p id=([0-9]{1,3})>(.*?)</p>", "$2" + Environment.NewLine);

                    File.WriteAllText(string.Format(paragraph_error_vi_path, j), content_vi);
                    File.WriteAllText(string.Format(paragraph_error_en_path, j), content_en);
                    j++;
                }
                catch
                { }
            }
        }

        //lấy para loại trừ các câu quá ngắn
        private void GetParagraphExceptMiniSentense()
        {
            string paragraph_done_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_id\{0}_p_en.txt";
            string paragraph_done_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_id\{0}_p_vi.txt";

            string paragraph_error_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_mini\{0}_p_en.txt";
            string paragraph_error_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_mini\{0}_p_vi.txt";

            string content_en = "";
            string content_vi = "";
            string output_en = "";
            string output_vi = "";
            int j = 0;

            for (int i = 0; i < 255; i++)
            {
                try
                {
                    content_en = File.ReadAllText(string.Format(paragraph_done_en_path, i));
                    content_vi = File.ReadAllText(string.Format(paragraph_done_vi_path, i));

                    WebBrowser webBrowser1 = new WebBrowser();
                    webBrowser1.Navigate("about:blank");
                    webBrowser1.Document.Write(content_vi);
                    var items_vi = webBrowser1.Document.GetElementsByTagName("p");

                    WebBrowser webBrowser2 = new WebBrowser();
                    webBrowser2.Navigate("about:blank");
                    webBrowser2.Document.Write(content_en);
                    var items_en = webBrowser2.Document.GetElementsByTagName("p");

                    content_vi = "";
                    content_en = "";

                    for (int m = 0; m < items_vi.Count; m++)
                    {
                        if ((items_vi[m].InnerText == null && items_en[m].InnerText == null) || (items_vi[m].InnerText.Length <= 30 && items_en[m].InnerText.Length <= 30))
                        {
                            output_en += String.Format("<p={0}>{1}</p>\n", m, items_en[m].InnerText);
                            output_vi += String.Format("<p={0}>{1}</p>\n", m, items_vi[m].InnerText);
                        }
                        else
                        {
                            content_vi += String.Format("<p={0}>{1}</p>\n", m, items_vi[m].InnerText);
                            content_en += String.Format("<p={0}>{1}</p>\n", m, items_en[m].InnerText);
                        }
                    }

                    File.WriteAllText(string.Format(paragraph_error_vi_path, j), content_vi);
                    File.WriteAllText(string.Format(paragraph_error_en_path, j), content_en);
                    j++;

                }
                catch { }
            }
            File.WriteAllText(string.Format(paragraph_error_vi_path, 1000), output_vi);
            File.WriteAllText(string.Format(paragraph_error_en_path, 1000), output_en);

        }

        //gộp tất cả file english lại thành 1 file để tách câu
        private void GetOneFile()
        {
            string paragraph_done_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_id_after_except\{0}_p_en.txt";
            string paragraph_done_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_id_after_except\{0}_p_vi.txt";
            string content_en = "";
            string content_vi = "";

            int j = 0;
            string out_en = "", out_vi = "";

            for (int i = 0; i < 254; i++)
            {
                try
                {
                    out_en = File.ReadAllText(string.Format(paragraph_done_en_path, i));
                    content_en += string.Format("<div class='file' id={0}>\r\n{1}\r\n</div>\r\n", i, out_en);

                    out_vi = File.ReadAllText(string.Format(paragraph_done_vi_path, i));
                    content_vi += string.Format("<div class='file' id={0}>\r\n{1}\r\n</div>\r\n", i, out_vi);
                }
                catch { }
            }
            content_en = Regex.Replace(content_en, "(<p id=\\d{1,3}>)", "$1\r\n");
            content_en = Regex.Replace(content_en, "(</p>)", "\r\n$1");
            File.WriteAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\onefile_en.txt", content_en);

            content_vi = Regex.Replace(content_vi, "(<p id=\\d{1,3}>)", "$1\r\n");
            content_vi = Regex.Replace(content_vi, "(</p>)", "\r\n$1");
            File.WriteAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\onefile_vi.txt", content_vi);
        }

        //chuyen file thanh tieng viet k dau va nguoc lai
        private void GetOneFile_KhongDau()
        {

            string content_en = "";
            string output_en = "";

            string[] vn = "y,ý,ỳ,ỷ,ỹ,ỵ,Y,Ý,Ỳ,Ỷ,Ỹ,Ỵ,a,á,à,ả,ạ,ã,â,ấ,ầ,ẩ,ẫ,ậ,ă,ắ,ằ,ẳ,ẵ,ặ,i,í,ì,ỉ,ĩ,ị,o,ó,ò,ỏ,õ,ọ,ô,ố,ồ,ổ,ỗ,ộ,ơ,ớ,ờ,ở,ỡ,ợ,e,é,è,ẻ,ẽ,ẹ,ê,ế,ề,ể,ễ,ệ,u,ú,ù,ủ,ũ,ụ,ư,ứ,ừ,ử,ữ,ự,đ,A,Á,À,Ả,Ạ,Ã,Â,Ấ,Ầ,Ẩ,Ẫ,Ậ,Ă,Ắ,Ằ,Ẳ,Ẵ,Ặ,I,Í,Ì,Ỉ,Ĩ,Ị,O,Ó,Ò,Ỏ,Õ,Ọ,Ô,Ố,Ồ,Ổ,Ỗ,Ộ,Ơ,Ớ,Ờ,Ở,Ỡ,Ợ,E,É,È,Ẻ,Ẽ,Ẹ,Ê,Ế,Ề,Ể,Ễ,Ệ,U,Ú,Ù,Ủ,Ũ,Ụ,Ư,Ứ,Ừ,Ử,Ữ,Ự,Đ".Split(',');
            string[] vn_1 = "@@y@@,@@ys@@,@@yf@@,@@yr@@,@@yx@@,@@yj@@,@@Y@@,@@YS@@,@@YF@@,@@YR@@,@@YX@@,@@YJ@@,a,@@as@@,@@af@@,@@ar@@,@@aj@@,@@ax@@,@@aa@@,@@aas@@,@@aaf@@,@@aar@@,@@aax@@,@@aaj@@,@@aw@@,@@aws@@,@@awf@@,@@awr@@,@@awx@@,@@awj@@,i,@@is@@,@@if@@,@@ir@@,@@ix@@,@@ij@@,o,@@os@@,@@of@@,@@or@@,@@ox@@,@@oj@@,@@oo@@,@@oos@@,@@oof@@,@@oor@@,@@oox@@,@@ooj@@,@@ow@@,@@ows@@,@@owf@@,@@owr@@,@@owx@@,@@owj@@,e,@@es@@,@@ef@@,@@er@@,@@ex@@,@@ej@@,@@ee@@,@@ees@@,@@eef@@,@@eer@@,@@eex@@,@@eej@@,u,@@us@@,@@uf@@,@@ur@@,@@ux@@,@@uj@@,@@uw@@,@@uws@@,@@uwf@@,@@uwr@@,@@uwx@@,@@uwj@@,@@dd@@,A,@@AS@@,@@AF@@,@@AR@@,@@AJ@@,@@AX@@,@@AA@@,@@AAS@@,@@AAF@@,@@AAR@@,@@AAX@@,@@AAJ@@,@@AW@@,@@AWS@@,@@AWF@@,@@AWR@@,@@AWX@@,@@AWJ@@,I,@@IS@@,@@IF@@,@@IR@@,@@IX@@,@@IJ@@,O,@@OS@@,@@OF@@,@@OR@@,@@OX@@,@@OJ@@,@@OO@@,@@OOS@@,@@OOF@@,@@OOR@@,@@OOX@@,@@OOJ@@,@@OW@@,@@OWS@@,@@OWF@@,@@OWR@@,@@OWX@@,@@OWJ@@,E,@@ES@@,@@EF@@,@@ER@@,@@EX@@,@@EJ@@,@@EE@@,@@EES@@,@@EEF@@,@@EER@@,@@EEX@@,@@EEJ@@,U,@@US@@,@@UF@@,@@UR@@,@@UX@@,@@UJ@@,@@UW@@,@@UWS@@,@@UWF@@,@@UWR@@,@@UWX@@,@@UWJ@@,@@DD@@".Split(',');

            content_en = File.ReadAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\new_ngulieu\onfile_vi.ANCII.encode.only.onefile");
            for (int i = 0; i < vn.Length; i++)
            {
                content_en = Regex.Replace(content_en, vn_1[i], vn[i]);
            }

            File.WriteAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\new_ngulieu\onfile_vi.UTF8.only.decode", content_en);
        }

        //sửa lại đuôi file là .en và .vi
        private void ChangeFileExtension()
        {
            string paragraph_done_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_id_after_except\{0}_p_en.txt";
            string paragraph_done_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_id_after_except\{0}_p_vi.txt";

            string paragraph_error_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_new\{0}_p.en";
            string paragraph_error_vi_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\graph_new\{0}_p.vi";

            string content_en = "";
            string content_vi = "";

            int j = 0;

            for (int i = 0; i <= 253; i++)
            {
                try
                {
                    content_en = File.ReadAllText(string.Format(paragraph_done_en_path, i));
                    content_vi = File.ReadAllText(string.Format(paragraph_done_vi_path, i));

                    File.WriteAllText(string.Format(paragraph_error_vi_path, j), content_vi);
                    File.WriteAllText(string.Format(paragraph_error_en_path, j), content_en);
                    j++;
                }
                catch
                { }
            }
        }

        //tạo file semcor sử dụng trong training senlearner
        private void CreateSemCorFile()
        {
            string[] allFiles_a = Directory.GetFiles(@"D:\LUAN VAN\TOOL\semcor1.6\semcor1.6\brown1\tagfiles");
            string[] allFiles_b = Directory.GetFiles(@"D:\LUAN VAN\TOOL\semcor1.6\semcor1.6\brown2\tagfiles");
            List<string> allFiles = new List<string>();
            allFiles.AddRange(allFiles_a);
            allFiles.AddRange(allFiles_b);

            string content = "";
            string output = "";
            foreach (string file in allFiles)
            {
                content = File.ReadAllText(file);
                output += content;
            }

            File.WriteAllText(@"D:\semcor3.0.all", output);
        }

        //loai het cac ki tu dac biet trong ascii
        private void LoaiBoKiTuDacBiet()
        {
            string ontent3 = File.ReadAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\onefile\onfile_en.ANCII.onlytext.txt", Encoding.UTF8);
            var inas = Regex.Matches(ontent3, "�");
            string output = Regex.Replace(ontent3, "�", " ");
            File.WriteAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\onefile\onfile_en.ANCII.onlytext.loaibokitudacbiet", output);
        }

        //loai bo het cac ki tu lon be de tranh gay hieu lam la tag
        private void LoaiBoKituLonBe()
        {
            string content = File.ReadAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\onefile\onefile_vi.txt", Encoding.UTF8);
            content = Regex.Replace(content, @"<p id=(\d+)>", "@@start@@p id=$1@@end@@");
            content = Regex.Replace(content, @"<div class='file' id=(\d+)>", "@@start@@div class='file' id=$1@@end@@");
            content = Regex.Replace(content, @"</div>", "@@start@@/div@@end@@");
            content = Regex.Replace(content, @"</p>", "@@start@@/p@@end@@");
            var x1 = Regex.Matches(content, "<");
            content = Regex.Replace(content, @"<", " ");
            var x2 = Regex.Matches(content, ">");
            content = Regex.Replace(content, @">", " ");
            content = Regex.Replace(content, "@@start@@", "<");
            content = Regex.Replace(content, "@@end@@", ">");
            File.WriteAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\onefile\onefile_vi.loaibokitulonbe", content);
        }

        //tap hop lai file cu sau khi loai bi ki tu dac biet trong ascii
        private void TapHopLaiFileCu()
        {
            string content = File.ReadAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\new_ngulieu\onfile_vi.ANCII.encode.only", Encoding.UTF8);
            content = Regex.Replace(content, "	", " @@tabtab@@ ");
            content = Regex.Replace(content, "@@tabtab@@ .+", " ");
            content = Regex.Replace(content, "\n", " ");
            content = Regex.Replace(content, " +", " ");
            content = Regex.Replace(content, "</p>", "\r\n</p>");
            File.WriteAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\new_ngulieu\onfile_vi.ANCII.encode.only.onefile", content);
        }

        //tap hop lai file cu nhung kem theo the sentense <span>
        private void TapHopLaiFileCu_Ver2()
        {
            string content = File.ReadAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\new_ngulieu\onfile_en.ANCII.only", Encoding.UTF8);
            //string content = File.ReadAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\new_ngulieu\test.txt", Encoding.UTF8);

            content = Regex.Replace(content, "	SENT", "</span><span>");
            content = Regex.Replace(content, "	", " @@tabtab@@ ");
            content = Regex.Replace(content, " @@tabtab@@ ([^%\r\n])+", " ");
            content = Regex.Replace(content, "@@tabtab@@", " ");
            content = Regex.Replace(content, "</p>", "</span></p>");
            content = Regex.Replace(content, @"<p id=(\d+)>", "<p id=$1><span>");
            content = Regex.Replace(content, "(\r\n|\n)", " ");
            content = Regex.Replace(content, " +", " ");
            content = Regex.Replace(content, "> +<", "><");
            content = Regex.Replace(content, "<span></span>", "");
            content = Regex.Replace(content, ">", ">\r\n");
            content = Regex.Replace(content, "<", "\r\n<");
            content = Regex.Replace(content, "(\r\n)+", "\r\n");
            File.WriteAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\new_ngulieu\onfile_en.ANCII.only.onefile", content);
        }

        //đưa ra nhiều file
        private void ExtractFiles()
        {
            string path_en = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\new_ngulieu\onfile_en.ANCII.span.txt";
            string path_vi = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\new_ngulieu\onfile_vi.UTF8.span.txt";

            string path_same = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\new_ngulieu\extract_same\{0}";
            string path_diff = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\new_ngulieu\extract_diff\{0}";

            string content_en = File.ReadAllText(path_en);
            string content_vi = File.ReadAllText(path_vi);

            WebBrowser webBrowser1 = new WebBrowser();
            webBrowser1.Navigate("about:blank");
            webBrowser1.Document.Write(content_en);
            var items_en = webBrowser1.Document.GetElementsByTagName("div");

            WebBrowser webBrowser2 = new WebBrowser();
            webBrowser2.Navigate("about:blank");
            webBrowser2.Document.Write(content_vi);
            var items_vi = webBrowser2.Document.GetElementsByTagName("div");

            for (int i = 0; i < items_en.Count; i++)
            {
                var p_en = items_en[i].GetElementsByTagName("p");
                var p_vi = items_vi[i].GetElementsByTagName("p");

                string output_en = Regex.Replace(items_en[i].OuterHtml, "(<P .+>)<SPAN>", "$1\r\n<SPAN>");
                output_en = Regex.Replace(output_en, "</SPAN>", "</SPAN>\r\n");
                string output_vi = Regex.Replace(items_vi[i].OuterHtml, "(<P .+>)<SPAN>", "$1\r\n<SPAN>");
                output_vi = Regex.Replace(output_vi, "</SPAN>", "</SPAN>\r\n");

                bool flag = true;
                for (int j = 0; j < p_en.Count; j++)
                {
                    var span_en = p_en[j].GetElementsByTagName("span");
                    var span_vi = p_vi[j].GetElementsByTagName("span");
                    if (span_en.Count != span_vi.Count)
                    {
                        flag = false;
                        break;
                    }
                }

                if (flag)
                {
                    File.WriteAllText(String.Format(path_same, i.ToString() + ".same.en"), output_en);
                    File.WriteAllText(String.Format(path_same, i.ToString() + ".same.vi"), output_vi);
                }
                else
                {
                    File.WriteAllText(String.Format(path_diff, i.ToString() + ".diff.en"), output_en);
                    File.WriteAllText(String.Format(path_diff, i.ToString() + ".diff.vi"), output_vi);
                }
            }
        }

        //kiem tra xem co sai sot gì k 
        private void ChuanHoa()
        {
            string path_en = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\final_corpus\fin\{0}.en";
            string path_vi = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\final_corpus\fin\{0}.vi";
            string path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\final_corpus\after_remove_(\";
            string output_en = "";
            string output_vi = "";
            for (int i = 0; i <= 253; i++)
            {
                try
                {
                    string content_en = File.ReadAllText(string.Format(path_en, i));
                    string content_vi = File.ReadAllText(string.Format(path_vi, i));
                    var ma_en = Regex.Matches(content_en, "([(].+?[)])");
                    var ma_vi = Regex.Matches(content_vi, "([(].+?[)])");


                    foreach (Match ma in ma_en)
                    {
                        output_en += ma.Groups[1].Value + Environment.NewLine;
                    }
                    foreach (Match ma in ma_vi)
                    {
                        output_vi += ma.Groups[1].Value + Environment.NewLine;
                    }
                    content_en = Regex.Replace(content_en, "([(].+?[)])", " ");
                    content_en = Regex.Replace(content_en, "[(|)]", " ");
                    content_vi = Regex.Replace(content_vi, "([(].+?[)])", " ");
                    content_vi = Regex.Replace(content_vi, "[(|)]", " ");

                    File.WriteAllText(path + i.ToString() + ".en", content_en);
                    File.WriteAllText(path + i.ToString() + ".vi", content_vi);
                }
                catch (Exception ex)
                {
                }
            }
            File.WriteAllText(@"D:\file_en.txt", output_en);
            File.WriteAllText(@"D:\file_vi.txt", output_vi);
        }

        //kiem tra lan cuoi
        private void TestFiles()
        {
            string path_en = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\final_corpus\after_remove_(\{0}.en";
            string path_vi = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\final_corpus\after_remove_(\{0}.vi";

            string rs1 = "", rs2 = "", rs3 = "";
            for (int i = 0; i <= 253; i++)
            {
                try
                {
                    string content_en = File.ReadAllText(string.Format(path_en, i));
                    string content_vi = File.ReadAllText(string.Format(path_vi, i));
                    int a = Regex.Matches(content_en, "<").Count;
                    int b = Regex.Matches(content_vi, "<").Count;
                    if (a != b)
                    {
                        rs1 += "; " + i.ToString();
                    }

                    a = Regex.Matches(content_en, ">").Count;
                    b = Regex.Matches(content_vi, ">").Count;
                    if (a != b)
                    {
                        rs2 += "; " + i.ToString();
                    }


                    a = Regex.Matches(content_en, @"<\").Count;
                    b = Regex.Matches(content_vi, @"<\").Count;
                    if (a != b)
                    {
                        rs3 += "; " + i.ToString();
                    }
                }
                catch { }
                { }
            }
            string output = "file <: " + rs1 + Environment.NewLine;
            output += "file >: " + rs2 + Environment.NewLine;
            output += @"file <\: " + rs3 + Environment.NewLine;
            File.WriteAllText(@"D:\output.txt", output);
        }

        //loai bo het cac tag
        private void RemoveAllTag()
        {
            string path_en = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\final_corpus\my_semcor\{0}.en";
            string path_vi = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\final_corpus\my_semcor\{0}.vi";
            string[] files = Directory.GetFiles(path_en);
            string rs_en = "", rs_vi = "", file_en = "", file_vi = "", error = "";

            for (int i = 0; i <= 249; i++)
            {


                string content_en = File.ReadAllText(string.Format(path_en, i));
                string content_vi = File.ReadAllText(string.Format(path_vi, i));

                WebBrowser webBrowser1 = new WebBrowser();
                webBrowser1.Navigate("about:blank");
                webBrowser1.Document.Write(content_en);
                var items_en = webBrowser1.Document.GetElementsByTagName("span");

                WebBrowser webBrowser2 = new WebBrowser();
                webBrowser2.Navigate("about:blank");
                webBrowser2.Document.Write(content_vi);
                var items_vi = webBrowser2.Document.GetElementsByTagName("span");

                file_en = "";
                file_vi = "";
                for (int j = 0; j < items_en.Count; j++)
                {
                    try
                    {
                        var item_en = items_en[j].InnerText;
                        var item_vi = items_vi[j].InnerText;
                        if (item_en == null || item_vi == null)
                            continue;
                        item_en = Regex.Replace(item_en, " +", " ");
                        item_vi = Regex.Replace(item_vi, " +", " ");

                        int a = Regex.Matches(item_en, " ").Count;
                        int b = Regex.Matches(item_vi, " ").Count;

                        if (a < 3 || b < 3)
                        {
                            rs_en += string.Format("<SPAN id={0}>{1}<SPAN>" + Environment.NewLine, (i * 1000 + j), item_en);
                            rs_vi += string.Format("<SPAN id={0}>{1}<SPAN>" + Environment.NewLine, (i * 1000 + j), item_vi);
                        }
                        else
                        {
                            file_en += item_en + "\r\n";
                            file_vi += item_vi + "\r\n";
                        }


                        File.WriteAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\final_corpus\no_tag\" + i.ToString() + ".en", file_en);
                        File.WriteAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\final_corpus\no_tag\" + i.ToString() + ".vi", file_vi);
                    }
                    catch (Exception ex)
                    {
                        File.AppendAllText(@"D:\error", i.ToString() + " " + ex.Message + Environment.NewLine);
                    }
                }


                //   File.WriteAllText(@"D:\error.txt", error);


            }
            File.WriteAllText(@"D:\en.txt", rs_en);
            File.WriteAllText(@"D:\vi.txt", rs_vi);
        }

        //tao ngu lieu thô
        private void CreateTagSemCor()
        {
            string path_en = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\final_corpus\after_remove_(\en\{0}.en";
            string path_vi = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\final_corpus\after_remove_(\vi\{0}.vi";
            string path_save_en = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\final_corpus\my_semcor\{0}.en";
            string path_save_vi = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\final_corpus\my_semcor\{0}.vi";
            string path_text_en = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\final_corpus\text\{0}.en";
            string path_text_vi = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\final_corpus\text\{0}.vi";
            string path_text_nograph_en = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\final_corpus\text_nograph\{0}.en";
            string path_text_nograph_vi = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\final_corpus\text_nograph\{0}.vi";
            int aaa = 0;
            int bbb = 0;
            string output_en = "", output_vi = "", content_vi = "", content_en = "", text_en = "", text_vi = "", text_nograph_vi = "", text_nograph_en = "";
            for (int i = 0; i <= 249; i++)
            {
                try
                {
                    content_vi = File.ReadAllText(string.Format(path_vi, i));
                    content_en = File.ReadAllText(string.Format(path_en, i));
                }
                catch
                {
                    continue;
                }
                WebBrowser webBrowser1 = new WebBrowser();
                webBrowser1.Navigate("about:blank");
                webBrowser1.Document.Write(content_en);
                var p_en = webBrowser1.Document.GetElementsByTagName("p");

                WebBrowser webBrowser2 = new WebBrowser();
                webBrowser2.Navigate("about:blank");
                webBrowser2.Document.Write(content_vi);
                var p_vi = webBrowser2.Document.GetElementsByTagName("p");


                if (p_en.Count != p_vi.Count)
                {
                    File.AppendAllText(@"D:\CreateTagSemCor", "the p k giong nhau " + i.ToString() + "\r\n");
                    continue;
                }
                else
                {
                    output_en = "<contextfile concordance='e-semcor'>\r\n<context filename=esc-" + i.ToString() + " paras=yes>\r\n";
                    output_vi = "<contextfile concordance='v-semcor'>\r\n<context filename=vsc-" + i.ToString() + " paras=yes>\r\n";
                    text_en = "";
                    text_vi = "";
                    text_nograph_en = "";
                    text_nograph_vi = "";
                    int snum = 0;
                    for (int j = 0; j < p_en.Count; j++)
                    {
                        var s_en = p_en[j].GetElementsByTagName("span");
                        var s_vi = p_vi[j].GetElementsByTagName("span");
                        aaa += s_en.Count;
                        bbb += s_vi.Count;
                        if (s_en.Count != s_vi.Count)
                        {
                            File.AppendAllText(@"D:\CreateTagSemCor", "the s k giong nhau " + i.ToString() + "\r\n");
                            break;
                        }
                        else
                        {
                            output_en += string.Format("<p pnum={0}>\r\n", j + 1);
                            output_vi += string.Format("<p pnum={0}>\r\n", j + 1);
                            text_en += "<br />\r\n";
                            text_vi += "<br />\r\n";

                            for (int k = 0; k < s_en.Count; k++)
                            {
                                snum++;
                                output_en += string.Format("<s snum={0}>{1}</s>\r\n", snum, s_en[k].InnerText);
                                output_vi += string.Format("<s snum={0}>{1}</s>\r\n", snum, s_vi[k].InnerText);
                                text_en += s_en[k].InnerText + "\r\n";
                                text_vi += s_vi[k].InnerText + "\r\n";
                                text_nograph_en += s_en[k].InnerText + "\r\n";
                                text_nograph_vi += s_vi[k].InnerText + "\r\n";
                            }
                            output_en += "</p>\r\n";
                            output_vi += "</p>\r\n";
                        }
                    }
                    output_en += "</context>\r\n</contextfile>\r\n";
                    output_vi += "</context>\r\n</contextfile>\r\n";
                    File.WriteAllText(string.Format(path_save_en, i), output_en);
                    File.WriteAllText(string.Format(path_save_vi, i), output_vi);
                    File.WriteAllText(string.Format(path_text_en, i), text_en);
                    File.WriteAllText(string.Format(path_text_vi, i), text_vi);
                    File.WriteAllText(string.Format(path_text_nograph_en, i), text_nograph_en);
                    File.WriteAllText(string.Format(path_text_nograph_vi, i), text_nograph_vi);
                }

            }
            MessageBox.Show(aaa.ToString() + ";b=" + bbb.ToString());
        }


        private void JoinFile()
        {
            string path_en = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\the_end\my_semcor\{0}.en";
            string path_vi = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\the_end\my_semcor\{0}.vi";
            string output_en = "", output_vi = "", content_vi, content_en;
            for (int i = 0; i <= 249; i++)
            {
                try
                {
                    content_vi = File.ReadAllText(string.Format(path_vi, i));
                    content_en = File.ReadAllText(string.Format(path_en, i));
                }
                catch
                { continue; }
                output_en += string.Format("<f id ={0}>{1}</f>", i, content_en);
                output_vi += string.Format("<f id ={0}>{1}</f>", i, content_vi);
            }
            File.WriteAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\the_end\onefile_notag.en", output_en);
            File.WriteAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\the_end\onefile_notag.vi", output_vi);
        }

        //tao cac file mau lien quan den tung pos
        private void CheckPos()
        {
            string pos = File.ReadAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\the_end\temp.txt");
            string file_en = File.ReadAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\the_end\pos_en.txt");
            string file_semcor = File.ReadAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\the_end\semcor3.0.all");

            string[] collection = Regex.Split(pos, "\r\n");


            ////
            List<string> list = new List<string>();
            foreach (string item in collection)
            {
                if (list.Contains(item) == false)
                    list.Add(item);
            }
            //string tree="", semcor="",all="";
            //foreach (string item in list)
            //{
            //    string pattern_tree = string.Format(".*?\t{0}\t(.+)",item);
            //    string pattern_semcor = string.Format(".*?={0} .*>(.+)<.*", item);
            //    if (Regex.IsMatch(file_en, pattern_tree))
            //    {
            //        if (Regex.IsMatch(file_semcor, pattern_tree))
            //        {
            //            all += item + "\r\n";
            //        }
            //        else
            //        {
            //            tree += item + "\r\n";
            //        }
            //    }
            //    else
            //    {
            //        if (Regex.IsMatch(file_semcor, pattern_tree))
            //        {
            //            semcor += item + "\r\n";
            //        }
            //    }
            //}


            List<string> items = new List<string>();
            string output_file = "", output_semcor = "";
            foreach (string item in list)
            {
                items.Clear();
                output_file = "";
                string pattern = string.Format(".*?\t{0}\t(.+)", item);
                var mas = Regex.Matches(file_en, pattern);
                int i = 0;
                foreach (Match ma in mas)
                {
                    if (!items.Contains(ma.Groups[1].Value))
                    {
                        items.Add(ma.Groups[1].Value);
                        output_file += ma.Value + "\r\n";
                        if (i++ == 100)
                            break;
                    }


                }

                items.Clear();
                i = 0;
                pattern = string.Format(".*?={0} .*>(.+)<.*", item);
                mas = Regex.Matches(file_semcor, pattern);
                output_semcor = "";
                foreach (Match ma in mas)
                {

                    if (!items.Contains(ma.Groups[1].Value))
                    {
                        items.Add(ma.Groups[1].Value);
                        output_semcor += ma.Value + "\r\n";
                        if (i++ == 100)
                            break;
                    }

                }
                File.WriteAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\the_end\" + item + "_file.txt", output_file);
                File.WriteAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\the_end\" + item + "_semcor.txt", output_semcor);
            }
        }


        //tach semcor (origin-chi chua cau) ra nhieu file
        private void SplitSemCorTemplate()
        {
            ////split semcor
            //string file_en = File.ReadAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\ver1\semcor.tag.txt");
            //string path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\ver1\files\{0}.txt";
            //var mas = Regex.Matches(file_en, "<s snum=(\\d+)>(.|\n)+?</s>\r\n");
            //string result = "";
            //var sss = mas[0];
            //foreach (Match ma in mas)
            //{
            //    result = Regex.Match(ma.Value, "<s snum=(\\d+)>(.|\n)+?</s>").Groups[1].Value;
            //    int count = Convert.ToInt32(result) / 100;
            //    File.AppendAllText(string.Format(path, count), ma.Value);
            //}


            //split postagger
            string postagger = File.ReadAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\ver1\pos.txt");
            string path_postagger = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\ver1\semcors\{0}.txt";
            var mas = Regex.Matches(postagger, "<contextfile concordance='e-semcor'>\r\n<context filename=esc-(\\d+) paras=yes>(.|\n)+?</contextfile>");
            string result = "";
            foreach (Match ma in mas)
            {
                result = Regex.Match(ma.Value, "filename=esc-(\\d+)").Groups[1].Value;
                int count = Convert.ToInt32(result);
                File.AppendAllText(string.Format(path_postagger, count), ma.Value);
            }
        }

        private void FinalSemCorEn()
        {
            string pos_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\ver1\semcors\{0}.txt";
            string semtag_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\ver1\files\{0}.txt";
            string semcors_en_path = @"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\ver1\final_semcors_en\semcor_en_{0}.tag";
            for (int i = 0; i <= 249; i++)
            {
                try
                {
                    string content = File.ReadAllText(string.Format(pos_path, i));
                    MatchCollection mas = Regex.Matches(content, "<s snum=(\\d+)>(.|\n)+?</s>");
                    foreach (Match ma in mas)
                    {
                        int number = Convert.ToInt32(ma.Groups[1].Value);
                        string sem_content = File.ReadAllText(string.Format(semtag_path, number / 100));
                        string sentence = Regex.Match(sem_content, string.Format("<s snum={0}>(.|\n)+?</s>", number)).Value;
                        content = content.Replace(ma.Value, sentence);
                    }
                    File.WriteAllText(string.Format(semcors_en_path, i), content);
                }
                catch (Exception ex)
                {
                }
            }

        }

        //tao file semcor de dua vao senseleaner
        private void CreateSemCorTemplate()
        {
            string file_en = File.ReadAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\thay toan\real\file.en.pos");
            var mas = Regex.Matches(file_en, "(.+)\r\n");
            string result = "";
            int count = 1;
            foreach (Match ma in mas)
            {
                result = "";
                var sub_ma = Regex.Match(ma.Value, "(.+?)\t(.+?)\t(.+)\r\n");
                if (sub_ma.Groups[1].Value == "")
                {
                    if (!ma.Value.Contains("<f") && !ma.Value.Contains("</f"))
                    {
                        if (ma.Value.Contains("<s "))
                        {
                            result = string.Format("<s snum={0}>\r\n", count);
                            count++;
                        }
                        else
                            result = ma.Value;

                    }
                    else
                        continue;
                }
                else
                {
                    if (sub_ma.Groups[3].Value == "<unknown>")
                    {
                        result = string.Format("<wf cmd=ignore pos={0}>{1}</wf>\r\n", sub_ma.Groups[2].Value, sub_ma.Groups[1].Value);
                    }
                    else
                    {
                        switch (sub_ma.Groups[2].Value)
                        {

                            case "CD":
                            case "RP":
                                result = string.Format("<wf cmd=done pos={0} ot=notag>{1}</wf>\r\n", sub_ma.Groups[2].Value, sub_ma.Groups[1].Value);
                                break;
                            case "JJ":
                            case "JJR":
                            case "JJS":
                                result = string.Format("<wf cmd=done pos=JJ lemma={0}>{1}</wf>\r\n", sub_ma.Groups[3].Value, sub_ma.Groups[1].Value);
                                break;
                            case "NN":
                            case "NNS":
                                result = string.Format("<wf cmd=done pos=NN lemma={0}>{1}</wf>\r\n", sub_ma.Groups[3].Value, sub_ma.Groups[1].Value);
                                break;
                            case "PP":
                                result = string.Format("<wf cmd=ignore pos=PRP>{0}</wf>\r\n", sub_ma.Groups[1].Value);
                                break;
                            case "PP$":
                                result = string.Format("<wf cmd=ignore pos=PRP$>{0}</wf>\r\n", sub_ma.Groups[1].Value);
                                break;
                            case "RB":
                            case "RBR":
                            case "RBS":
                                result = string.Format("<wf cmd=done pos=RB lemma={0}>{1}</wf>\r\n", sub_ma.Groups[3].Value, sub_ma.Groups[1].Value);
                                break;
                            case "SENT":
                                result = string.Format("<punc>{0}</punc>\r\n", sub_ma.Groups[1].Value);
                                break;
                            case "SYM":
                            case "#":
                            case "$":
                            case ",":
                            case "\"":
                            case "``":
                            case ":":
                            case "''":
                            case ")":
                            case "(":
                                result = string.Format("<wf cmd=ignore pos=SYM>{0}</wf>\r\n", sub_ma.Groups[1].Value);
                                break;
                            case "VB":
                            case "VBD":
                            case "VBG":
                            case "VBN":
                            case "VBP":
                            case "VBZ":
                            case "VH":
                            case "VHD":
                            case "VHG":
                            case "VHN":
                            case "VHP":
                            case "VHZ":
                            case "VV":
                            case "VVD":
                            case "VVG":
                            case "VVN":
                            case "VVP":
                            case "VVZ":
                                result = string.Format("<wf cmd=done pos=VB lemma={0}>{1}</wf>\r\n", sub_ma.Groups[3].Value, sub_ma.Groups[1].Value);
                                break;
                            case "WP":
                            case "WRB":
                            case "WP$":
                            case "WDT":
                            case "CC":
                            case "DT":
                            case "EX":
                            case "FW":
                            case "IN":
                            case "LS":
                            case "MD":
                            case "NP":
                            case "NPS":
                            case "PDT":
                            case "POS":
                            case "TO":
                            case "UH":
                                result = string.Format("<wf cmd=ignore pos={0}>{1}</wf>\r\n", sub_ma.Groups[2].Value, sub_ma.Groups[1].Value);
                                break;
                            default:
                                File.AppendAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\thay toan\real\file.en.beforesemcor", sub_ma.Value);
                                result = sub_ma.Value;
                                break;
                        }
                    }
                }
                File.AppendAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\thay toan\real\file.en.beforesemcor", result);
            }
        }

        //tao file chua cap cau tu ngu lieu thay toan cho
        private void GetSentences()
        {
            string file = File.ReadAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\thay toan\real\EVC-20K.aligned");
            var matches = Regex.Matches(file, "srcsent:(.+)\ntgtsent:(.+)\naligned:(.+)\n");
            int i = 0;
            foreach (Match ma in matches)
            {
                i++;
                File.AppendAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\thay toan\real\file.en", string.Format("<s snum={0}>{1}</s>\r\n", i, ma.Groups[1].Value));
                File.AppendAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\thay toan\real\file.vi", string.Format("<s snum={0}>{1}</s>\r\n", i, ma.Groups[2].Value));
                File.AppendAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\thay toan\real\file.file", string.Format("<s snum={0}>{1}</s>\r\n", i, ma.Value));
            }
        }

        //kiem tra file sau khi treetagger xem co trung voi file cu k => trung 100%
        private void test()
        {
            string file = File.ReadAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\thay toan\real\file.en.pos");
            var mas = Regex.Matches(file, "(.+)\r\n");
            foreach (Match ma in mas)
            {
                var sub = Regex.Match(ma.Value, "(.+)?\t.+\t.+");
                if (sub.Groups[1].Value == "")
                {
                    File.AppendAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\thay toan\real\file.en.pos.test", "\r\n");
                }
                else
                {
                    File.AppendAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\thay toan\real\file.en.pos.test", sub.Groups[1].Value + " ");
                }
            }
        }

        private void GetFinalSemcorVi()
        {
            string semcor_en_path = @"D:\CreateCorpus\files\l_thay toan\real\semcor.en.lexicon.txt";
            string file_align_path = @"D:\CreateCorpus\files\l_thay toan\real\file.file";
            string semcor_vi_path = @"D:\CreateCorpus\files\l_thay toan\real\file.vi.semcor";

            string semcor_en = File.ReadAllText(semcor_en_path);
            string file_align = File.ReadAllText(file_align_path);

            int count0=0, count2 = 0;

            var mas_semcor = Regex.Matches(semcor_en, @"<s snum=(\d+)>((.|\n)+?)</s>");
            foreach (Match ma_semcor in mas_semcor)
            {
                int snum = Convert.ToInt32(ma_semcor.Groups[1].Value);
                Match ma_align = Regex.Match(file_align, string.Format("<s snum={0}>srcsent:(.+)\ntgtsent:(.+)\naligned:(.+)\n</s>", snum));
                List<string> sen_en = ma_align.Groups[1].Value.Trim().Split(' ').ToList<string>(); sen_en.Insert(0, "");
                List<string> sen_vi = ma_align.Groups[2].Value.Trim().Split(' ').ToList<string>(); sen_vi.Insert(0, "");
                string str_align = ma_align.Groups[3].Value.Replace(" ", "");

                File.AppendAllText(semcor_vi_path, string.Format("<s snum={0}>\r\n", snum));

                for (int i = 1; i < sen_vi.Count; i++)
                {
                    if (sen_vi[i] == ".")
                    { }
                    string word_vi = Regex.Replace(sen_vi[i], @"([[\^$.|?*+(){}])", "[$1]");
                    if (word_vi == "[^]") word_vi = "\\^";
                    Match find_same = Regex.Match(ma_semcor.Value,string.Format("<.+?>{0}</.+?>\r\n",word_vi));
                    if (find_same.Value != "")
                    {
                        File.AppendAllText(semcor_vi_path, find_same.Value);
                    }
                    else
                    {
                        MatchCollection numbers_align = Regex.Matches(str_align, string.Format(@"[(](\d+),{0}[)]", i));
                        //if (numbers_align.Count == 0)
                        //{
                        //    File.AppendAllText(semcor_vi_path, string.Format("<wf cmd=ignore pos={0}>{1}</wf>\r\n", "UNKNOWN", sen_vi[i]));
                        //    count0++;
                        //}
                        //else if (numbers_align.Count > 1)
                        //{
                        //    File.AppendAllText(semcor_vi_path, string.Format("<wf cmd=ignore pos={0}>{1}</wf>\r\n", "UNKNOWN", sen_vi[i]));
                        //    count2++;
                        //}
                        //else
                        //{
                            int number_en = Convert.ToInt32(numbers_align[0].Groups[1].Value);

                            sen_en[number_en] = Regex.Replace(sen_en[number_en], @"([[\^$.|?*+(){}])", "[$1]");
                            if (sen_en[number_en] == "[^]") sen_en[number_en] = "\\^";
                            Match ma_tag = Regex.Match(ma_semcor.Value, string.Format(@"<wf .+?>{0}</wf>\r\n", sen_en[number_en]));
                            string rs = Regex.Replace(ma_tag.Value, "<(wf .+?)>(.+)</wf>", "<$1>" + sen_vi[i] + "</wf>");
                            if (Regex.IsMatch(rs, "lemma=[?]") != true)
                            {
                                rs = Regex.Replace(rs, "<(.+?)lemma=([^ ]+?)>(.+)</wf>", "<$1lemma=" + sen_vi[i] + ">" + sen_vi[i] + "</wf>");
                                rs = Regex.Replace(rs, "<(.+?)lemma=(.+?) (.+?)>(.+)</wf>", "<$1lemma=" + sen_vi[i] + " $3>" + sen_vi[i] + "</wf>");
                            }
                            File.AppendAllText(semcor_vi_path, rs);
                       // }
                    }
                }

                File.AppendAllText(semcor_vi_path, "</s>\r\n");
            }
            File.AppendAllText(@"D:\count0count2.txt", "count0 = " + count0.ToString() + " ; count2 = " + count2.ToString());
        }

        private void Get1()
        {
            string file = File.ReadAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\thay toan\pos\file_pos.vi");
            var mas = Regex.Matches(file, "(.+?)/(.+?) ");
            foreach (Match ma in mas)
            {
                if(!ma.Groups[1].Value.Contains("<s") && (!ma.Groups[1].Value.Contains("snum=")))
                {
                    File.AppendAllText(@"C:\Documents and Settings\bRea.K\My Documents\Visual Studio 2010\Projects\CreateCorpus\files\thay toan\pos\file_pos.vi.get1",ma.Groups[1].Value+"\t"+ma.Groups[2].Value);
                }
            }
        }

        private void KiemtraLoaiTu()
        {
            string semcor_en_path = @"D:\CreateCorpus\files\l_thay toan\real\semcor.en.lexicon.txt";
            string file_align_path = @"D:\CreateCorpus\files\l_thay toan\real\file.file";
            string semcor_vi_path = @"D:\CreateCorpus\files\l_thay toan\real\file.vi.semcor";

            //string semcor_en = File.ReadAllText(semcor_en_path);
            //string file_align = File.ReadAllText(file_align_path);

            //var mas_semcor = Regex.Matches(semcor_en, @"<s snum=(\d+)>((.|\n)+?)</s>");
            //foreach (Match ma_semcor in mas_semcor)
            //{
            //    var mas_word = Regex.Matches(ma_semcor.Groups[2].Value, "<wf .+? lexsn=(.+?)>(.+?)</wf>");

            //    int snum = Convert.ToInt32(ma_semcor.Groups[1].Value);
            //    Match ma_align = Regex.Match(file_align, string.Format("<s snum={0}>srcsent:(.+)\ntgtsent:(.+)\naligned:(.+)\n</s>", snum));
            //    List<string> sen_en = ma_align.Groups[1].Value.Trim().Split(' ').ToList<string>(); sen_en.Insert(0, "");
            //    List<string> sen_vi = ma_align.Groups[2].Value.Trim().Split(' ').ToList<string>(); sen_vi.Insert(0, "");
            //    string str_align = ma_align.Groups[3].Value.Replace(" ", "");

            //    foreach (Match ma_word in mas_word)
            //    {
            //        int index = sen_en.IndexOf(ma_word.Groups[2].Value);
            //        MatchCollection numbers_align = Regex.Matches(str_align, string.Format(@"[(]{0},(\d+)[)]", index));
            //        File.AppendAllText(@"D:\test.txt", "\r\n"+ma_word.Groups[2].Value+ " " +numbers_align.Count.ToString()+":\t");
            //        foreach (Match number_align in numbers_align)
            //        {
            //            string vi_word = sen_vi[Convert.ToInt32(number_align.Groups[1].Value)];
            //            File.AppendAllText(@"D:\test.txt", vi_word + " , ");
            //        }
            //    }

               
            //}

            string semcor_en = File.ReadAllText(semcor_en_path);
            string file_align = File.ReadAllText(file_align_path);
            int count0=0, count2 = 0;

            var mas_semcor = Regex.Matches(file_align, @"<s snum=(\d+)>srcsent:(.+)\ntgtsent:(.+)\naligned:(.+)\n</s>");
            foreach (Match ma_semcor in mas_semcor)
            {
                List<string> sen_en = ma_semcor.Groups[2].Value.Trim().Split(' ').ToList<string>(); sen_en.Insert(0, "");
                List<string> sen_vi = ma_semcor.Groups[3].Value.Trim().Split(' ').ToList<string>(); sen_vi.Insert(0, "");
                string str_align = ma_semcor.Groups[4].Value.Replace(" ", "");
                string sentence = Regex.Match(semcor_en,(string.Format( @"<s snum={0}>((.|\r|\n)+?)</s>",ma_semcor.Groups[1].Value))).Groups[1].Value;

                //File.AppendAllText(@"D:\test_semcor_vi.txt", string.Format(@"<s snum={0}>", ma_semcor.Groups[1].Value));
                foreach (string word in sen_vi)
                {
                    int index_vi = sen_vi.IndexOf(word);
                    MatchCollection numbers_align = Regex.Matches(str_align, string.Format(@"[(](\d+),{0}[)]", index_vi));

                    
                    if (numbers_align.Count == 0)
                    {
                       // File.AppendAllText(@"D:\error_test_semcor_vi.txt", string.Format("snum={0} ----- {1}", ma_semcor.Groups[1].Value,word));
                        File.AppendAllText(@"D:\test_semcor_vi.txt", string.Format(@"<wf cmd=ignore>{0}</wf>", word));
                        count0++;
                    }
                    else if (numbers_align.Count > 1)
                    {
                        count2++;
                    }
                    else
                    {
                        string en_word = sen_en[Convert.ToInt32(numbers_align[0].Groups[1].Value)];
                    }
                    
                }
               // File.AppendAllText(@"D:\test_semcor_vi.txt", string.Format(@"</s>", ma_semcor.Groups[1].Value));

            }
        }
    }
}